'''

大众点评

'''
import requests,time
import random,re
from config import *
from pyquery import PyQuery as pq
from helper import Helper
from mongo import MongoClient

class Crawler( object ):

    def __init__(self):

        self.fonts = Helper.getCssLink()
        self.mongo = MongoClient()

    def get_index(self,page):

        r = requests.get( url ="http://www.dianping.com/shenyang/ch10/r1780p"+str(page)+"" ,headers=public_for_header )

        if r.status_code == 200:

            # 禁止HTML语言格式化
            html = r.text.replace("&#x","").replace(";",'')

            self.parse_index ( html )

    def parse_index( self,html ):

        doc = pq(html)
        # 获取标题
        titles = doc(".shop-list .tit a h4").items()
        # 获取星级
        stars = doc(".sml-rank-stars").items()
        # 获取评价数量
        comments = doc(".comment").items()
        #评价列表
        comments_list = doc(".comment-list").items()
        # 基础信息
        base_info = doc(".shop-list ul li").items()
        # 推荐菜
        recommend = doc(".recommend").items()
        # 所在地址
        addresses = doc(".J_operate").items()

        for k, v, z ,l,b,r,a in zip(titles, stars, comments,comments_list,base_info,recommend,addresses):

            nums = (z("a b").eq(0).text()).split("\n")

            avgs = (z("a b").eq(1).text()).split("\n")

            # 口味
            flavors = ( l("span b").eq(0).text().split("\n"))
            # 环境
            envs = ( l("span b").eq(1).text().split("\n"))
            #服务
            services = ( l("span b").eq(2).text().split("\n"))

            nus = []
            ars = []
            flas = []
            enn = []
            sers = []
            for num in (nums):

                # 字典匹配结果
                for font in self.fonts:

                    if font["name"] == num :

                        if font["id"] == "11":

                            result = 0

                            nus.append(str(result))

                        else:

                            result = int( font["id"] ) - 1

                            nus.append( str(result ))

                        break

                    elif num != font["name"] and (num == "1" ):

                        loc = nums.index( num )

                        nus.insert(loc, "1")

                        break

                    else:

                        continue

            # 匹配人均价钱
            for avg in (avgs):

                # 字典匹配结果
                for font in self.fonts:

                    if font["name"] == avg:

                        if font["id"] == "11":

                            result = 0

                            ars.append(str(result))

                        else:

                            result = int(font["id"]) - 1

                            ars.append(str(result))

                        break

                    elif avg != font["name"] and (avg == "1"):

                        loc = avgs.index(avg)

                        ars.insert(loc, "1")

                        break

                    elif avg != font["name"] and (avg == "￥1"):

                        ars.insert(0, "1")

                        break

                    else:

                        continue
            # 匹配口味评分
            for flavor in flavors:

                for font in self.fonts:

                    if font["name"] == flavor:

                        if font["id"] == "11":

                            result = "0"

                            flas.append(str(result))

                        else:

                            result = int(font["id"]) - 1

                            flas.append(str(result))

                        break

                    elif flavor != font["name"] and (flavor == ".1"):

                        loc = flavors.index( flavor )

                        flas.insert(loc, "1")

                        break

                    else:

                        continue
            # 匹配环境
            for env in envs:

                for font in self.fonts:

                    if font["name"] == env:

                        if font["id"] == "11":

                            result = "0"

                            enn.append(str(result))

                        else:

                            result = int(font["id"]) - 1

                            enn.append(str(result))

                        break

                    elif env != font["name"] and (env == ".1"):

                        loc = envs.index( env )

                        enn.insert(loc, "1")

                        break

                    else:

                        continue
            #匹配服务
            for sr in services:

                for font in self.fonts:

                    if font["name"] == sr:

                        if font["id"] == "11":

                            result = "0"

                            sers.append(str(result))

                        else:

                            result = int(font["id"]) - 1

                            sers.append(str(result))


                        break

                    elif sr != font["name"] and (sr == ".1"):

                        loc = services.index( sr )

                        sers.insert(loc, "1")

                        break

                    else:

                        continue

            # 匹配地址
            addr = re.search(r'data-address="(.*?)"', str(a)).group(1)

            data = {

                #店铺id
                "shop_id":b("a").attr("data-shopid"),
                # 店铺标题
                "title": k.text(),
                # 链接地址
                "href":b("a").attr("href"),
                # 图片地址
                "image":b("a img ").attr("src"),
                # 评论查看地址
                "comments":b("a").attr("href") + "#comment",
                # 评价星级
                "star":v.attr("title"),
                # 推荐菜品
                "recommend":r("a").text().split(" "),
                # 商家地址
                "address":addr,
                # 点评数量
                "commentNumber":''.join( nus ),
                # 人均消费
                "avgPer":''.join( ars ),
                # 口味评分
                "flavor":int(''.join( flas ))/10,
                # 环境评分
                "environment":int(''.join( enn ))/10,
                # 服务评分
                "service":int(''.join( sers ))/10,
            }
            # 写入mongoDB数据库
            self.mongo.saveMongo( data )


    # 增加访问页数
    def main(self):

        for i in range(1,20):

            self.get_index( i )

            return


if __name__ == '__main__':

    c = Crawler()

    c.main()
